﻿#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2015-10-14 03:51:01
# Project: alter



import re
import string
from pyspider.libs.base_handler import *
from pyspider.database.mysql.mysqldb import SQL
from pyspider.libs.login import Login


class Handler(BaseHandler):
    #针对百度手机助手（最具人气的应用商店）开发的爬虫程序
	#author：titan52
	#date：2015/10/14

    @every(minutes=24 * 60)
    def on_start(self):
        login = Login()
        url = 'http://shouji.baidu.com/software/list?cid=504&boardid=board_100_018&page_num='
        for i in range(1,9):
            crawl_url = url + str(i)
            print crawl_url
            self.crawl(crawl_url,  callback=self.index_page)

    @config(age=10 * 24 * 60 * 60)
    def index_page(self, response):
        for each in response.doc('a[href^="http://shouji.baidu.com/software/item?"]').items():
            self.crawl(each.attr.href, priority=9, callback=self.detail_page)
                
    @config(priority=2)
    def detail_page(self, response):
        s = response.doc('.detail > .size').text()
        s = s[s.index(':')+1:s.index('M')]
        #size = string.atof(s)
        v = response.doc('.detail > .version').text()
        v = v[v.index(':')+1:]
        n = response.doc('.detail > .download-num').text()
        
        star = response.doc('#doc > div.yui3-g > div > div.app-intro > div > div.content-right > div.app-feature > span.star-xbig > span').attr('style')
        star = star[star.index(':')+1:star.index('%')]
        #star = string.atof(star)
        
        #find(str,pos_start,pos_end)#返回值：如果查到：返回查找的第一个出现的位置。否则，返回-1
        if n.find(u'万') != -1:
            n = n[n.index(':')+1:n.index(u'万')]
            number = string.atoi(n)*10000
            number = str(number)
        else:
            n = n[n.index(':')+1:]
            number = n
            #number = string.atoi(n)

        comment = response.doc('.app-detail > .section-container > .section-head  > .head-content').text()
        
        intro = response.doc('.brief-long ').text()
        d_url = response.doc('.area-download > .apk').attr.href
        
        return {
            "name": response.doc('.app-name > span').text(),
            "number":number,
            "size":s,
            #version 保留字符串格式
            "version":v,			
            "star" : star,
            "comment":comment,
            "intro":intro,
            "url":response.url,
            "d_url": d_url,
        }
    
    def on_result(self, result):
        if not result or not result['name']:
            return
        print 'ready for inser result.'
        sql=SQL()
        sql.replace('baidu',**result)
        #test = Test()
        #test.replace('baidu',**result)
        print 'insert mysql database done.'